{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## [stars-list](https://github.com/anlzou/stars-list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-02T07:42:36.008772Z",
     "start_time": "2020-09-02T07:42:36.005112Z"
    }
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from requests_html import HTMLSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {\n",
    "    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',\n",
    "    'accept-encoding': 'gzip, deflate, br',\n",
    "    'accept-language': 'zh-CN,zh;q=0.9',\n",
    "    'cache-control': 'no-cache',\n",
    "    'pragma': 'no-cache',\n",
    "    'sec-fetch-dest': 'document',\n",
    "    'sec-fetch-mode': 'navigate',\n",
    "    'sec-fetch-site': 'same-site',\n",
    "    'sec-fetch-user': '?1',\n",
    "    'upgrade-insecure-requests': '1',\n",
    "    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',\n",
    "}\n",
    "\n",
    "'''\n",
    "    data[0]:types\n",
    "    data[1]:types_url\n",
    "    list_title:project name\n",
    "    list_title_url:project url\n",
    "    list_text:project brief introduction\n",
    "'''\n",
    "data = []\n",
    "github_url = \"https://github.com\"\n",
    "session = HTMLSession()\n",
    "\n",
    "list_title = []\n",
    "list_title_url = []\n",
    "list_text = []\n",
    "\n",
    "\n",
    "def get_title_and_text(url):\n",
    "    resp = session.get(url, headers=headers)\n",
    "    url = resp.html.xpath('//div[@class=\"BtnGroup\"]/a[@rel=\"nofollow\"]/@href')\n",
    "    button = resp.html.xpath(\n",
    "        '//div[@class=\"BtnGroup\"]/a[@rel=\"nofollow\"]/text()')\n",
    "\n",
    "    if len(url) == 0 or (len(url) == 1 and button[0] == \"Next\"):\n",
    "        title = resp.html.xpath(\n",
    "            '//div[@class=\"d-inline-block mb-1\"]/h3/a/@href')\n",
    "        title_url = [github_url + i for i in title]\n",
    "        text = resp.html.xpath('//div[@class=\"py-1\"]/p/text()')\n",
    "        text = [j.replace(\"\\n      \", \"\")\n",
    "                for j in [i.replace(\"\\n        \", \"\") for i in text]]\n",
    "\n",
    "        list_title.append(title)\n",
    "        list_title_url.append(title_url)\n",
    "        list_text.append(text)\n",
    "\n",
    "    elif len(pages_next) == 2:\n",
    "        title = resp.html.xpath(\n",
    "            '//div[@class=\"d-inline-block mb-1\"]/h3/a/@href')\n",
    "        title_url = [github_url + i for i in title]\n",
    "        text = resp.html.xpath('//div[@class=\"py-1\"]/p/text()')\n",
    "        text = [j.replace(\"\\n      \", \"\")\n",
    "                for j in [i.replace(\"\\n        \", \"\") for i in text]]\n",
    "\n",
    "        list_title.append(title)\n",
    "        list_title_url.append(title_url)\n",
    "        list_text.append(text)\n",
    "\n",
    "        resp = session.get(pages_next[1], headers=headers)\n",
    "        pages_next = resp.html.xpath(\n",
    "            '//div[@class=\"BtnGroup\"]/a[@rel=\"nofollow\"]/@href')\n",
    "        get_title_and_text(resp)\n",
    "\n",
    "\n",
    "def get_types(username):\n",
    "    stars_url = \"https://github.com/\" + username + \"?tab=stars\"\n",
    "    resp = session.get(stars_url, headers=headers)\n",
    "\n",
    "    # 获取类型\n",
    "    types = resp.html.xpath('//ul[@class=\"filter-list\"]/li/a/text()')\n",
    "    types_result = [k.replace(\"  \", \"\") for k in [j.replace(\n",
    "        \"\\n      \", \"\") for j in [i.replace(\"\\n        \", \"\") for i in types]]]\n",
    "    types_result = types_result[4:]  # 去掉多余\n",
    "\n",
    "    # 获取类型链接\n",
    "    urls = resp.html.xpath('//ul[@class=\"filter-list\"]/li/a/@href')\n",
    "    urls_result = urls[4:]  # 去掉多余\n",
    "\n",
    "    return types_result, urls_result\n",
    "\n",
    "\n",
    "def makeMarkdown():\n",
    "    name = \"## Stars-List\"\n",
    "    text = \"> A curated list of my GitHub stars! Generated by [stars-list](https://github.com/anlzou/stars-list)\"\n",
    "    details_head_contents = \"<details><summary>Contents</summary>\"\n",
    "    details_head_List = \"<details><summary>List</summary>\"\n",
    "    datails_font = \"</details>\"\n",
    "\n",
    "#     添加Contents\n",
    "    md_text = name+\"\\n\"+text+\"\\n\\n\"+details_head_contents+\"\\n\\n\"\n",
    "    for i in range(len(data[0])):\n",
    "        li = \"- [\" + str(data[0][i]) + \"](##\" + str(data[0][i]) + \")\\n\"\n",
    "        md_text += li\n",
    "    md_text += datails_font+\"\\n\\n\"\n",
    "\n",
    "#     添加List\n",
    "    md_text += details_head_List+\"\\n\\n\"\n",
    "    for i in range(len(list_title)):\n",
    "        li_title = \"## \" + str(data[0][i]) + \"\\n\"\n",
    "        li_text = \"\"\n",
    "        for j in range(len(list_title[i])):\n",
    "            li_text += \"- [\" + str(list_title[i][j][1:]) + \\\n",
    "                \"](\" + str(list_title_url[i][j]) + \")\"\n",
    "            if len(str(list_text[i][j])) != 0:\n",
    "                li_text += \" - \"\n",
    "                li_text += str(list_text[i][j]) + \"\\n\"\n",
    "        md_text += li_title\n",
    "        md_text += li_text\n",
    "    md_text += datails_font\n",
    "\n",
    "    md_file = open('StarsList.md', mode='w+', encoding='utf-8')\n",
    "    md_file.write(md_text)\n",
    "    md_file.close()\n",
    "\n",
    "\n",
    "def run(username):\n",
    "    type_data = get_types(username)\n",
    "    for i in type_data:\n",
    "        data.append(i)\n",
    "\n",
    "#     每个类型的页面\n",
    "    for url in data[1]:\n",
    "        get_title_and_text(url)\n",
    "\n",
    "#     生成markdown文件\n",
    "    makeMarkdown()\n",
    "    print(\"finish\")\n",
    "\n",
    "\n",
    "'''\n",
    "    todo：\n",
    "    获取项目名称、评论、stars、fork\n",
    "    模板制作：显示样式，是否需要stars&fork\n",
    "    支持输入文件名，可有可无\".md\"\n",
    "'''\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    username = input(\"input your github username: \")\n",
    "    print(\"wait a moment...\")\n",
    "    run(username)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
